import urllib3
import requests
import lxml.html
import pymongo

import lianjia


class Products(lianjia.Base):

    def _load_city(self):
        districts = []
        response: urllib3.HTTPResponse = self._http.request('GET', self._url_formatter % (self._city, '/ershoufang'))
        if response.status != 200:
            return []

        document: lxml.html.HtmlElement = lxml.html.fromstring(response.data)
        elements = document.cssselect('div.position > dl:nth-child(2) > dd > div:nth-child(1) > div > a')
        for element in elements:  # type: lxml.html.HtmlElement
            districts.append(element.attrib['href'])
        return districts

    def _load_district(self, district):
        regions = []
        response: urllib3.HTTPResponse = self._http.request('GET', self._url_formatter % (self._city, district))
        if response.status != 200:
            return []

        document: lxml.html.HtmlElement = lxml.html.fromstring(response.data)
        elements = document.cssselect('div.position > dl:nth-child(2) > dd > div:nth-child(1) > div:nth-child(2) > a')
        for element in elements:  # type: lxml.html.HtmlElement
            regions.append(element.attrib['href'])
        return regions

    def _load_region(self, region):
        response: urllib3.HTTPResponse = self._http.request('GET', self._url_formatter % (self._city, region))
        if response.status != 200:
            return 0

        document: lxml.html.HtmlElement = lxml.html.fromstring(response.data)
        elements = document.cssselect('div.page-box.fr > div.page-box.house-lst-page-box a')  # type: list[lxml.html.HtmlElement]
        if len(elements) > 0:
            elements.reverse()
            for element in elements:
                page = element.text  # type: str
                if page.isdigit():
                    return int(page)

        return 100

    def _load_apartments(self, region, page):
        apartments = []
        response: urllib3.HTTPResponse = self._http.request('GET', self._url_formatter % (self._city, region + 'pg' + page + '/'))
        if response.status != 200:
            return []

        document: lxml.html.HtmlElement = lxml.html.fromstring(response.data)
        elements = document.cssselect('ul.sellListContent > li')  # type: list[lxml.html.HtmlElement]
        for element in elements:
            apartment={}

            elements_title = element.cssselect('div.title > a')  # type: list[lxml.html.HtmlElement]
            if len(elements_title) == 1:
                elements_title_first = elements_title[0]
                if 'data-housecode' in elements_title_first.attrib:
                    apartment['_id'] = elements_title_first.attrib['data-housecode']
                else:
                    code = elements_title_first.attrib['href']  # type: str
                    index = code.rfind('/')
                    if index >= 0:
                        code = code[index+1:]
                    index = code.rfind('.')
                    if index >= 0:
                        code = code[:index]
                    apartment['_id'] = code
                apartment['description'] = elements_title_first.text

            element_address = element.cssselect('div.address > div.houseInfo')  # type: list[lxml.html.HtmlElement]
            if len(element_address) == 1:
                element_apartment = element_address[0].cssselect('a')  # type: list[lxml.html.HtmlElement]
                if len(element_apartment) == 1:
                    apartment_href = element_apartment[0].attrib['href']  # type: str
                    if apartment_href[-1] == '/':
                        apartment_href = apartment_href[:-1]
                    index = apartment_href.rfind('/')
                    if index >= 0:
                        apartment_href = apartment_href[index+1:]
                    apartment['apartmentId'] = apartment_href
                address = element_address[0].text_content()  # type: str
                information = address.split('|')
                if len(information) > 0:
                    apartment['apartmentName'] = information[0].strip()
                    if len(information) > 2:
                        apartment['area'] = ''.join(filter(lambda ch: ch in '0123456789.', information[2].strip()))
                        if len(information) > 4:
                            apartment['decoration'] = information[4].strip()
                            if len(information) > 5:
                                apartment['elevator'] = information[5].strip() == '有电梯'

            element_total_price = element.cssselect('div.totalPrice > span')  # type: list[lxml.html.HtmlElement]
            if len(element_total_price) == 1:
                apartment['totalPrice'] = float(element_total_price[0].text)

            element_unit_price = element.cssselect('div.unitPrice > span')  # type: list[lxml.html.HtmlElement]
            if len(element_unit_price) == 1:
                apartment['unitPrice'] = float(''.join(filter(lambda ch: ch in '0123456789.', element_unit_price[0].text)))

            apartments.append(apartment)

        return apartments

    def run(self):
        client = pymongo.MongoClient(host='localhost', port=27017)
        db = client.lianjia
        collection: pymongo.collection.Collection = db.products

        districts = self._load_city()
        for district in districts:
            regions = self._load_district(district)
            for region in regions:
                page_count = self._load_region(region)
                for page in range(1, page_count + 1):
                    apartments = self._load_apartments(region, str(page))
                    if len(apartments) == 0:
                        break

                    apartments_to_insert = []
                    for apartment in apartments:
                        new = apartment.copy()
                        new.pop('_id')
                        if not collection.find_one_and_update({'_id': apartment['_id']}, {'$set': new}):
                            apartments_to_insert.append(apartment)
                    if len(apartments_to_insert) > 0:
                        collection.insert_many(apartments_to_insert)

                    if self._callback:
                        if not self._callback(apartments):
                            return
                    else:
                        pass
